# Replication Archive for
# Coppock, Alexander. Generalizing from Survey Experiments Conducted on Mechanical Turk: A Replication Approach
# Forthcoming in Political Science Research and Methods

rm(list = ls())

# Uncomment to set working directory to replication archive
# setwd("")

# Uncomment to install packages
# install.packages("tidyverse")
# install.packages("stargazer")
# install.packages("devtools")
# devtools::install_github("acoppock/commarobust")

library(tidyverse)
library(commarobust)
library(stargazer)

load("coppock_generalizability_studies.rdata")

# Study 1: Concealed Carry ------------------------------------------------

concealed_carry <-
  concealed_carry_stacked %>%
  split(.$sample) %>%
  map( ~ lm(dv_gun_s ~ Z_gun, weights = weights, data = .))

concealed_carry <- concealed_carry[c("original", "mt")]

sink("coppock_generalizability_online_appendix_table_1.txt")
stargazer(
  concealed_carry,
  se = makerobustseslist(concealed_carry),
  p = makerobustpslist(concealed_carry),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c("Citizens' Rights Frame", "Constant (Public Safety Frame)"),
  column.labels = c("Original", "MTurk"),
  dep.var.labels = rep("Support for Concealed Carry Law", 2),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Gun Control Original and Replication Results",
  label = "tab:guncontrol"
)
sink()


# Study 2: Immigration ----------------------------------------------------

immigration_1 <-
  immigration_stacked %>%
  filter(Z_brader_pos_neg != "control") %>%
  split(.$sample) %>%
  map( ~ lm(
    brader_num_imm_s ~ Z_brader_pos_neg,
    weights = weights,
    data = .
  ))

immigration_2 <-
  immigration_stacked %>%
  filter(Z_brader_pos_neg != "control") %>%
  split(.$sample) %>%
  map( ~ lm(
    brader_neg_impact_s ~ Z_brader_pos_neg,
    weights = weights,
    data = .
  ))


immigration_1 <- immigration_1[c("original", "mt")]
immigration_2 <- immigration_2[c("original", "mt")]
immigration <- c(immigration_1, immigration_2)

sink("coppock_generalizability_online_appendix_table_2.txt")
stargazer(
  immigration,
  se = makerobustseslist(immigration),
  p = makerobustpslist(immigration),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c("Positive Frame", "Constant (Negative Frame)"),
  column.labels = c("Original", "MTurk", "Original", "MTurk"),
  dep.var.labels = c("Support for Immigration", "Negative Impact"),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Immigration Original and Replication Results",
  label = "tab:immigration",
  column.sep.width = "0pt")
sink()

# Study 3: Death Penalty --------------------------------------------------

death_penalty <-
  death_penalty_stacked %>%
  filter(Z_CP_3 != "control") %>%
  split(.$sample) %>%
  map( ~ lm(CP_dv_s ~ Z_CP_3, weights = weights, data = .))

death_penalty <- death_penalty[c("original", "mt")]

sink("coppock_generalizability_online_appendix_table_3.txt")
stargazer(
  death_penalty,
  se = makerobustseslist(death_penalty),
  p = makerobustpslist(death_penalty),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  column.labels = c("Original",  "MTurk"),
  covariate.labels = c("African Americans", "Constant (Control)"),
  dep.var.labels = rep("Favor Death Penalty", 2),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Death Penalty Original and Replication Results",
  label = "tab:deathpenalty",
  column.sep.width = "0pt")
sink()

# Study 4: Superordinate Identity -----------------------------------------

superordinate_id <-
  superordinate_id_stacked %>%
  split(.$sample) %>%
  map( ~ lm(willing_s ~ Z_identity + Z_particularism, data = .))

superordinate_id <- superordinate_id[c("original", "mt")]

sink("coppock_generalizability_online_appendix_table_4.txt")
stargazer(
  superordinate_id,
  se = makerobustseslist(superordinate_id),
  p = makerobustpslist(superordinate_id),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c("Other Americans", "Public Schools", "Constant"),
  column.labels = c("Original", "MTurk"),
  dep.var.labels = c("Willingness to Pay Tax"),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Superordinate Identity Original and Replication Results",
  label = "tab:superordinateID")
sink()

# Study 5: Patriot Act ----------------------------------------------------

patriot_act <-
  patriot_act_stacked %>%
  split(.$sample) %>%
  map( ~ lm(
    PA_support_s ~ T1_condition_name,
    weights = weights,
    data = .
  ))

patriot_act <- patriot_act[c("original", "mt")]

sink("coppock_generalizability_online_appendix_table_5.txt")
stargazer(
  patriot_act,
  se = makerobustseslist(patriot_act),
  p = makerobustpslist(patriot_act),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c(
    "Con / Memory Based",
    "Con / No Processing",
    "Con / Online Processing",
    "Pro / Memory Based",
    "Pro / No Processing",
    "Pro / Online Processing",
    "Both / Memory Based",
    "Both / No Processing",
    "Both / Online Processing",
    "Constant (Control)"
  ),
  column.labels = c("Original", "MTurk"),
  dep.var.labels = c("Patriot Act Support"),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Patriot Act Original and Replication Results",
  label = "tab:patriotact")
sink()

# Study 6: Elite Endorsements ---------------------------------------------

elite_endorsements_stacked %>%
  filter(Z_imm_match != "control") %>%
  with(., table(sample))
elite_endorsements_stacked %>%
  filter(Z_fore_match != "control") %>%
  with(., table(sample))


# Note: must control for (or condition on) pid because of dif. probs. of assignment.

elite_endorsements_results_1 <-
  elite_endorsements_stacked %>%
  filter(Z_imm_match != "control") %>%
  mutate(Z_match = Z_imm_match) %>%
  split(.$sample) %>%
  map( ~ lm(imm_dv_s ~ Z_match + pid_3, weights = weights, data = .))

elite_endorsements_results_1 <-
  elite_endorsements_results_1[c("original", "mt")]

elite_endorsements_results_2 <-
  elite_endorsements_stacked %>%
  filter(Z_fore_match != "control") %>%
  mutate(Z_match = Z_fore_match) %>%
  split(.$sample) %>%
  map( ~ lm(fore_dv_s ~ Z_match + pid_3, weights = weights, data = .))

elite_endorsements_results_2 <-
  elite_endorsements_results_2[c("original", "mt")]

elite_endorsements <-
  c(elite_endorsements_results_1,
    elite_endorsements_results_2)


sink("coppock_generalizability_online_appendix_table_6.txt")
stargazer(
  elite_endorsements,
  se = makerobustseslist(elite_endorsements),
  p = makerobustpslist(elite_endorsements),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c("In Party Cue", "Republican", "Constant"),
  column.labels = c("Original", "MTurk", "Original", "MTurk"),
  dep.var.labels = c("Support for Immigration Bill", "Support for Foreclosure Bill"),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Elite Endorsements Original and Replication Results: Immigration",
  label = "tab:eliteendorsements",
  column.sep.width = "0pt")
sink("coppock_generalizability_online_appendix_table_1.txt")

# Study 7: Mental Illness -------------------------------------------------

mental_illness_1 <-
  mental_illness_stacked %>%
  split(.$sample) %>%
  map(
    ~ lm(
      mcginty_magazines_s ~ Z_mcginty_news + Z_mcginty_policy,
      weights = weights,
      data = .
    )
  )

mental_illness_2 <-
  mental_illness_stacked %>%
  split(.$sample) %>%
  map(
    ~ lm(
      mcginty_SMI_danger_s ~ Z_mcginty_news + Z_mcginty_policy,
      weights = weights,
      data = .
    )
  )

mental_illness_1 <- mental_illness_1[c("original", "mt")]
mental_illness_2 <- mental_illness_2[c("original", "mt")]

mental_illness <-
  c(mental_illness_1,
    mental_illness_2)


sink("coppock_generalizability_online_appendix_table_7.txt")
stargazer(
  mental_illness,
  se = makerobustseslist(mental_illness),
  p = makerobustpslist(mental_illness),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c("News", "LCM Ban", "Mental Illness", "Constant (Control)"),
  column.labels = c("Original", "MTurk", "Original", "MTurk"),
  dep.var.labels = c("Magazines", "SMI Danger"),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Mental Illness Original and Replication Results",
  label = "tab:mental_illness",
  column.sep.width = "0pt")
sink()

# Study 8: System Threat --------------------------------------------------

system_threat_1 <-
  system_threat_stacked %>%
  split(.$sample) %>%
  map( ~ lm(craig_num_imm_s ~ Z_craig, weights = weights, data = .))

system_threat_2 <-
  system_threat_stacked %>%
  split(.$sample) %>%
  map( ~ lm(craig_wol_s ~ Z_craig, weights = weights, data = .))

system_threat_1 <- system_threat_1[c("original", "mt")]
system_threat_2 <- system_threat_2[c("original", "mt")]
system_threat <- c(system_threat_1, system_threat_2)


sink("coppock_generalizability_online_appendix_table_8.txt")
stargazer(
  system_threat,
  se = makerobustseslist(system_threat),
  p = makerobustpslist(system_threat),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c("Majority Minority", "Constant (Placebo)"),
  column.labels = c("Original", "MTurk", "Original", "MTurk"),
  dep.var.labels = c("Support for Immigration", "Way of Life"),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "System Threat Original and Replication Results",
  label = "tab:system_threat",
  column.sep.width = "0pt")
sink()

# Study 9: Expert Economists ----------------------------------------------

expert_economists_1 <-
  expert_economists_stacked %>%
  split(.$sample) %>%
  map( ~ lm(agree_1_s ~ Z_local, weights = weights, data = mutate(., Z_local = Z_expert_1)))

expert_economists_2 <-
  expert_economists_stacked %>%
  split(.$sample) %>%
  map( ~ lm(agree_2_s ~ Z_local, weights = weights, data = mutate(., Z_local = Z_expert_2)))

expert_economists_3 <-
  expert_economists_stacked %>%
  split(.$sample) %>%
  map( ~ lm(agree_3_s ~ Z_local, weights = weights, data = mutate(., Z_local = Z_expert_3)))

expert_economists_4 <-
  expert_economists_stacked %>%
  split(.$sample) %>%
  map( ~ lm(agree_4_s ~ Z_local, weights = weights, data = mutate(., Z_local = Z_expert_4)))

expert_economists_5 <-
  expert_economists_stacked %>%
  split(.$sample) %>%
  map( ~ lm(agree_5_s ~ Z_local, weights = weights, data = mutate(., Z_local = Z_expert_5)))

expert_economists_1 <- expert_economists_1[c("original", "mt")]
expert_economists_2 <- expert_economists_2[c("original", "mt")]
expert_economists_3 <- expert_economists_3[c("original", "mt")]
expert_economists_4 <- expert_economists_4[c("original", "mt")]
expert_economists_5 <- expert_economists_5[c("original", "mt")]


expert_economists_a <-
  c(
    expert_economists_1,
    expert_economists_2,
    expert_economists_3
  )
expert_economists_b <-
  c(
    expert_economists_4,
    expert_economists_5
  )

sink("coppock_generalizability_online_appendix_table_9.txt")
stargazer(
  expert_economists_a,
  se = makerobustseslist(expert_economists_a),
  p = makerobustpslist(expert_economists_a),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c("Expert Treatment", "Constant (Control)"),
  column.labels = c("Original", "MTurk", "Original", "MTurk", "Original", "MTurk"),
  dep.var.labels = c(
    "Agree on Immigration",
    "Agree on Health Care",
    "Agree on Trade with China"
  ),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Expert Economists: Immigration, Health Care, and China",
  label = "tab:experts_1",
  column.sep.width = "0pt")
sink()


sink("coppock_generalizability_online_appendix_table_10.txt")
stargazer(
  expert_economists_b,
  se = makerobustseslist(expert_economists_b),
  p = makerobustpslist(expert_economists_b),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c("Expert Treatment", "Constant (Control)"),
  column.labels = c("Original", "MTurk", "Original", "MTurk"),
  dep.var.labels = c("Agree on Tax Cut", "Agree on Gold Standard"),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Expert Economists: Tax Cuts and Gold Standard",
  label = "tab:experts_2", 
  column.sep.width = "0pt")
sink()

# Studies 10 and 11: Free Trade -------------------------------------------

free_trade <-
  free_trade_stacked %>%
  split(.$sample) %>%
  map( ~ lm(
    Y_Hiscox_s ~ Z_Hiscox_expert + Z_Hiscox_valence,
    weights = weights,
    data = .
  ))


free_trade <- free_trade[c("original", "gfk", "mt")]

sink("coppock_generalizability_online_appendix_table_11.txt")
stargazer(
  free_trade,
  se = makerobustseslist(free_trade),
  p = makerobustpslist(free_trade),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c(
    "Expert",
    "Positive",
    "Negative",
    "Pos + Neg",
    "Constant (Control)"
  ),
  column.labels = c("Original", "GfK" , "MTurk"),
  dep.var.labels = rep("Support for Free Trade", 3),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Free Trade Original and Replication Results",
  label = "tab:freetrade",
  column.sep.width = "0pt")
sink()

# Studies 12 and 13: Frame Breadth ----------------------------------------

frame_breadth_1 <-
  frame_breadth_stacked %>%
  split(.$sample) %>%
  map( ~ lm(
    Y_crime_s ~ Z_crime_Y_crime,
    weights = weights,
    data = .
  ))
frame_breadth_2 <-
  frame_breadth_stacked %>%
  split(.$sample) %>%
  map( ~ lm(
    Y_health_s ~ Z_health_Y_health,
    weights = weights,
    data = .
  ))
frame_breadth_3 <-
  frame_breadth_stacked %>%
  split(.$sample) %>%
  map( ~ lm(
    Y_stimulus_s ~ Z_stimulus_Y_stimulus,
    weights = weights,
    data = .
  ))
frame_breadth_4 <-
  frame_breadth_stacked %>%
  split(.$sample) %>%
  map( ~ lm(
    Y_terror_s ~ Z_terror_Y_terror,
    weights = weights,
    data = .
  ))


frame_breadth_1 <- frame_breadth_1[c("original", "gfk", "mt")]
frame_breadth_2 <- frame_breadth_2[c("original", "gfk", "mt")]
frame_breadth_3 <- frame_breadth_3[c("original", "gfk", "mt")]
frame_breadth_4 <- frame_breadth_4[c("original", "gfk", "mt")]


sink("coppock_generalizability_online_appendix_table_12.txt")
stargazer(
  frame_breadth_1,
  se = makerobustseslist(frame_breadth_1),
  p = makerobustpslist(frame_breadth_1),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c("Crime Argument", "Constant (Control)"),
  column.labels = c("Original", "GfK" , "MTurk"),
  dep.var.labels = rep("Support for Crime Spending", 3),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Crime Spending Original and Replication Results",
  label = "tab: breadth_crime",
  column.sep.width = "0pt")
sink()

sink("coppock_generalizability_online_appendix_table_13.txt")
stargazer(
  frame_breadth_2,
  se = makerobustseslist(frame_breadth_2),
  p = makerobustpslist(frame_breadth_2),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c("Health Care Argument", "Constant (Control)"),
  column.labels = c("Original", "GfK" , "MTurk"),
  dep.var.labels = rep("Support for Health Care Spending", 3),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Health Care Spending Original and Replication Results",
  label = "tab: breadth_health",
  column.sep.width = "0pt")
sink()

sink("coppock_generalizability_online_appendix_table_14.txt")
stargazer(
  frame_breadth_3,
  se = makerobustseslist(frame_breadth_3),
  p = makerobustpslist(frame_breadth_3),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c("Stimulus Argument", "Constant (Control)"),
  column.labels = c("Original", "GfK" , "MTurk"),
  dep.var.labels = rep("Support for Stimulus Spending", 3),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Stimulus Spending Original and Replication Results",
  label = "tab: breadth_stimulus",
  column.sep.width = "0pt")
sink()


sink("coppock_generalizability_online_appendix_table_5.txt")
stargazer(
  frame_breadth_4,
  se = makerobustseslist(frame_breadth_4),
  p = makerobustpslist(frame_breadth_4),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c("Terrorism Argument", "Constant (Control)"),
  column.labels = c("Original", "GfK" , "MTurk"),
  dep.var.labels = rep("Support for Terrorism Spending", 3),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Terrorism Spending Original and Replication Results",
  label = "tab: breadth_terror",
  column.sep.width = "0pt")
sink()



# Studies 14 and 15: Polarization -----------------------------------------

polarization_1 <-
  polarization_stacked %>%
  filter(Z_Levendusky != "placebo") %>%
  split(.$sample) %>%
  map( ~ lm(L_ex_s ~ Z_Levendusky, weights = weights, data = .))

polarization_2 <-
  polarization_stacked %>%
  filter(Z_Levendusky != "placebo") %>%
  split(.$sample) %>%
  map( ~ lm(L_dif_s ~ Z_Levendusky, weights = weights, data = .))

polarization_1 <- polarization_1[c("original", "gfk", "mt")]
polarization_2 <- polarization_2[c("original", "gfk", "mt")]


sink("coppock_generalizability_online_appendix_table_16.txt")
stargazer(
  polarization_1,
  se = makerobustseslist(polarization_1),
  p = makerobustpslist(polarization_1),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c(
    "Polarizied Treatment",
    "Constant (Moderate)"
  ),
  column.labels = c("Original", "GfK", "MTurk"),
  dep.var.labels = rep("Extremity of Policy Views", 3),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Extremity Original and Replication Results",
  label = "tab: polarization_extremity",
  column.sep.width = "0pt")
sink()


sink("coppock_generalizability_online_appendix_table_17.txt")
stargazer(
  polarization_2,
  se = makerobustseslist(polarization_2),
  p = makerobustpslist(polarization_2),
  style = "apsr",
  omit.stat = c("f", "ser", "adj.rsq"),
  covariate.labels = c(
    "Polarizied Treatment",
    "Constant (Moderate)"
  ),
  column.labels = c("Original", "GfK", "MTurk"),
  dep.var.labels = rep("Perceived Polarization", 3),
  notes = c(
    "Robust standard errors are in parentheses.",
    "Dependent variable measured in standard units."
  ),
  title = "Perceived Polarization Original and Replication Results",
  label = "tab: polarization_perceived",
  column.sep.width = "0pt")
sink()


